#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence
@file: gsxt_offline_parse.py
@time: 2018/1/20 10:27
"""
import click

from common import util
from common.mongo import MongDb
from common.table_manage import TableManage
from config.app_conf import PROVINCE_LIST
from config.beanstalk_conf import BEANSTALK_CONF
from config.mongo_conf import MONGO_DB_TARGET
from ext.beanstalk_handler import BeanstalkHandler
from logger import Logger

log = Logger('gsxt_offline_parse.log').get_logger()


class OfflineParse(object):
    ALL_PROVINCE = 'all'

    def __init__(self, province, parse_tube, parse_status, beanstalk_conf, _log):

        self.log = _log
        self.parse_tube = parse_tube
        self.province = province
        self.parse_status = parse_status

        # 网页库
        self.__target_db = MongDb(MONGO_DB_TARGET['host'], MONGO_DB_TARGET['port'], MONGO_DB_TARGET['db'],
                                  MONGO_DB_TARGET['username'],
                                  MONGO_DB_TARGET['password'], log=log)
        self.province_list = self.get_province_list()

        self.beanstalk_thread = BeanstalkHandler(beanstalk_conf, log=_log)
        self.beanstalk_thread.start()

        self.log.info("解析消息队列: tube = {}".format(self.parse_tube))
        self.log.info("解析标识: parse_status = {}".format(self.parse_status))
        self.log.info("初始化离线解析完成...")

    # 开始解析单个省份
    def start_parse_province(self, province):

        # 获得网页库表名
        count = 0
        result_list = []
        web_page_table = TableManage.get_detail_page_table(province)
        self.log.info("当前解析省份: province = {} table = {}".format(province, web_page_table))
        for item in self.__target_db.traverse_batch(web_page_table,
                                                    {'parse_status': {'$ne': self.parse_status}},
                                                    batch_size=50):
            count += 1
            # 发送解析数据到消息队列
            self.beanstalk_thread.push_msg(self.parse_tube, item)

            company = item.get("_id")
            result_list.append({
                '_id': company,
                'parse_status': self.parse_status,
                'parse_status_time': util.get_now_time()
            })
            if len(result_list) >= 500:
                self.__target_db.insert_batch_data(web_page_table, result_list)
                del result_list[:]

            self.log.info("当前解析进度: province = {} count = {} company = {}".format(
                province, count, company))

        if len(result_list) > 0:
            self.__target_db.insert_batch_data(web_page_table, result_list)
            del result_list[:]

        self.log.info("当前省份解析完成: province = {}".format(province))

    # 创建索引
    def create_index(self, province):
        self.log.info("开始创建解析标志索引: province = {}".format(province))
        web_page_table = TableManage.get_detail_page_table(province)
        self.__target_db.create_index(web_page_table, [('parse_status', MongDb.ASCENDING)])
        self.log.info("创建解析标志索引完成: province = {}".format(province))

    # 启动解析
    def start(self):
        self.log.info("启动解析...")

        for province in self.province_list:
            # 确认索引存在
            self.create_index(province)
            # 开始解析
            self.start_parse_province(province)

        self.log.info("解析完成...")

    # 获得省份列表
    def get_province_list(self):
        if self.province == self.ALL_PROVINCE:
            return PROVINCE_LIST
        if self.province in PROVINCE_LIST:
            return [self.province]

        return []


@click.command()
@click.option('--province',
              default='all',
              help='需要解析的省份，gsxt代表全部省份')
@click.option('--parse_tube',
              default='gsxt_detail_parse',
              help='解析消息队列')
@click.option('--parse_status',
              type=int,
              default=1,
              help='解析省份的标识')
def main(province, parse_tube, parse_status):
    try:
        OfflineParse(province, parse_tube, parse_status, BEANSTALK_CONF, log).start()
    except Exception as e:
        log.error("离线解析异常:")
        log.exception(e)


if __name__ == '__main__':
    main()
